#!/bin/sh
#
# This script (along with the .travis.yml file) is used by Travis CI to
# build Zeek and run the tests.
#
# This script can also be used outside of Travis (the "all" build step is
# especially convenient in this case).  Note that if you use this script
# outside of Travis then you will need to fetch the private tests manually
# (if you don't, then the private tests will be skipped).

LEAK_TEST_DISTRO="ubuntu_18.04_leaktest"
COVERITY_DISTRO="ubuntu_18.04"

usage() {
    echo "usage: $0 CMD DISTRO"
    echo "  CMD is a build step:"
    echo "    install: install prereqs"
    echo "    build: build zeek"
    echo "    build_leaktests: build zeek with memory leak testing enabled"
    echo "    run: run the tests"
    echo "    all: do all of the above"
    echo "  DISTRO is a Linux distro, 'travis' to run without docker, or 'coverity' to run a coverity scan"
}

if [ $# -ne 2 ]; then
    usage
    exit 1
fi

step=$1
distro=$2

case $step in
    install) ;;
    build) ;;
    run) ;;
    all) ;;
    *) echo "Error: unknown build step: $step"; usage; exit 1 ;;
esac


# Install the coverity tools.
install_coverity() {
    rm -rf coverity_tool.tgz coverity-tools cov-analysis*

    echo "Downloading coverity tools..."
    wget -nv https://scan.coverity.com/download/cxx/linux64 --post-data "token=${COV_TOKEN}&project=Bro" -O coverity_tool.tgz
    tar xzf coverity_tool.tgz
    rm coverity_tool.tgz
    mv cov-analysis* coverity-tools
}


# Build Zeek with the coverity tools.
build_coverity() {
    # Cleanup any previous build (this is really only necessary if running this
    # outside of Travis).
    make distclean > /dev/null

    ./configure --prefix=`pwd`/build/root --enable-debug --disable-broker-tests --disable-python --disable-zeekctl

    export PATH=`pwd`/coverity-tools/bin:$PATH
    cd build
    cov-build --dir cov-int make -j 4
    cd ..
}


# Create a tar file and send it to coverity.
run_coverity() {
    EMAIL=zeek-commits-internal@zeek.org
    FILE=myproject.tgz
    VER=`cat VERSION`
    DESC=`git rev-parse HEAD`

    cd build
    echo "Creating tar file and sending to coverity..."
    tar czf ${FILE} cov-int
    curl --form token=${COV_TOKEN} --form email=${EMAIL} --form file=@${FILE} --form "version=${VER}" --form "description=${DESC}" https://scan.coverity.com/builds?project=Bro
}


# Create a docker container, and install all packages needed to build Zeek.
install_in_docker() {
    local_distro=$distro
    case $distro in
        centos_7)
        distro_cmds="yum -y install gdb make gcc gcc-c++ flex bison libpcap-devel openssl-devel git openssl which centos-release-scl && yum -y install devtoolset-7 && yum -y install epel-release && yum -y install cmake3"
        ;;
        debian_9)
        distro_cmds="apt-get update; apt-get -y install gdb cmake make flex bison python libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils clang-7 libc++-7-dev libc++abi-7-dev; update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100; update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100"
        ;;
        fedora_30)
        distro_cmds="yum -y install gdb cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel git sqlite findutils which zlib-devel; ln -s /usr/bin/python3 /usr/local/bin/python"
        ;;
        ubuntu_16.04)
        distro_cmds="apt-get update; apt-get -y install gdb cmake make flex bison python libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils wget xz-utils; wget -q https://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz; mkdir /clang-9; tar --strip-components=1 -C /clang-9 -xvf clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz; update-alternatives --install /usr/bin/cc cc /clang-9/bin/clang 100; update-alternatives --install /usr/bin/c++ c++ /clang-9/bin/clang++ 100"
        ;;
        ubuntu_18.04)
        distro_cmds="apt-get update; apt-get -y install wget xz-utils gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils; ln -s /usr/bin/python3 /usr/local/bin/python"
        ;;
        ${LEAK_TEST_DISTRO})
        distro_cmds="apt-get update; apt-get -y install gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils google-perftools libgoogle-perftools4 libgoogle-perftools-dev; ln -s /usr/bin/python3 /usr/local/bin/python"
        local_distro="ubuntu_18.04"
        ;;
        *)
        echo "Error: distro ${distro} is not recognized by this script"
        exit 1
        ;;
    esac

    docker_image=`echo $local_distro | tr '_' ':'`
    docker run --name zeektest -id -v "`pwd`:/zeek" -w /zeek ${docker_image} sh
    docker exec zeektest sh -c "${distro_cmds}"
}


# Build Zeek in a docker container.
build_in_docker() {
    recursed_distro=travis

    if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
        recursed_distro=coverity
    fi

    # Pass the distro as a different environment variable name to docker since
    # the script will set $distro to "travis" as part of the invocation.
    docker exec -e COV_TOKEN -e BUILD_DISTRO=${distro} zeektest sh ci/travis-job build ${recursed_distro}
}


# Run Zeek tests in a docker container.
run_in_docker() {
    recursed_distro=travis

    if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
        recursed_distro=coverity
    fi

    # Pass the distro as a different environment variable name to docker since
    # the script will set $distro to "travis" as part of the invocation.
    docker exec -t -e TRAVIS -e TRAVIS_PULL_REQUEST -e TESTING_PRIVATE_DEPLOYKEY -e COV_TOKEN -e BUILD_DISTRO=${distro} zeektest sh ci/travis-job run ${recursed_distro}
}

update_env() {
    if [ "${BUILD_DISTRO}" = "centos_7" ]; then
        source /opt/rh/devtoolset-7/enable
    elif [ "${BUILD_DISTRO}" = "debian_9" ]; then
        export CXXFLAGS="-stdlib=libc++"
    elif [ "${BUILD_DISTRO}" = "ubuntu_16.04" ]; then
        export CXXFLAGS="-stdlib=libc++"
        export LD_LIBRARY_PATH=/clang-9/lib
    fi
}

# Build Zeek.
build() {
    # Cleanup any previous build (this is really only necessary if running this
    # outside of Travis).
    make distclean > /dev/null

    update_env

    # Skip building Broker tests, python bindings, and zeekctl, as these are
    # not needed by the Zeek tests. If the distro is set for leak tests, enable
    # those options as well.
    if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
        ./configure --build-type=Release --disable-broker-tests --enable-cpp-tests --disable-python --disable-zeekctl && make -j 2
    else
        echo "Configuring zeek to build for leak testing"
        ./configure --build-type=Debug --disable-broker-tests --disable-python --disable-zeekctl --enable-perftools --enable-perftools-debug && make -j 2
    fi
}

# Get the private tests.
get_private_tests() {
    if [ "${TRAVIS}" != "true" ]; then
        # When not running in the Travis environment, just skip trying to get
        # the private tests.
        echo "Note: skipping private tests (to run them, do a git clone of the private testing repo in the 'testing/external' directory before running this script)."
    elif [ -n "$TESTING_PRIVATE_DEPLOYKEY" ]; then
        echo "$TESTING_PRIVATE_DEPLOYKEY" > travis_key.b64
        base64 --decode travis_key.b64 > travis_key
        rm travis_key.b64
        chmod 600 travis_key
        mkdir -p ~/.ssh
        mv travis_key ~/.ssh/id_rsa
        echo "Host *" >> ~/.ssh/config
        echo "    StrictHostKeyChecking no" >> ~/.ssh/config
        chmod 400 ~/.ssh/config
        git clone git@github.com:zeek/zeek-testing-private
        rm ~/.ssh/id_rsa
    elif [ -n "${TRAVIS_PULL_REQUEST}" ] && [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then
        # For pull request builds, the private key is not available, so skip
        # the private tests to avoid failing.
        echo "Note: skipping private tests because encrypted env. variables are not available in pull request builds."
    else
        echo "Error: cannot get private tests because encrypted env. variables are not defined."
        exit 1
    fi
}


# Run Zeek tests.
run() {
    update_env

    ulimit -c unlimited
    ulimit -a
    ret=0

    if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
        echo
        echo "Running unit tests ##################################################"
        echo

        set +e
        ( cd build && . ./zeek-path-dev.sh && zeek --test )

        if [ $? -ne 0 ]; then
            ret=1
        fi

        set -e
    fi

    echo
    echo "Running baseline tests ##############################################"
    echo
    cd testing/btest

    set +e
    # Must specify a value for "-j" option, otherwise Travis uses a huge value.
    if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
        ../../aux/btest/btest -j 4 -d
    else
        ../../aux/btest/btest -j 4 -d -g leaks
    fi

    if [ $? -ne 0 ]; then
        ret=1
    fi

    set -e

    echo
    echo "Getting external tests ##############################################"
    echo
    cd ../external

    if [ ! -d zeek-testing ]; then
        make init
    fi

    if [ -d zeek-testing ]; then
        commit=`cat commit-hash.zeek-testing`
        echo "Checking out $commit"
        ( cd zeek-testing && git checkout -q $commit )
    fi

    echo

    if [ ! -d zeek-testing-private ]; then
        get_private_tests
    fi

    if [ -d zeek-testing-private ]; then
        commit=`cat commit-hash.zeek-testing-private`
        echo "Checking out $commit"
        ( cd zeek-testing-private && git checkout -q $commit )
    fi

    echo
    echo "Running external tests ##############################################"
    echo

    set +e

    if [ -d zeek-testing ]; then
        cd zeek-testing
        if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
            make
        else
            make leaks
        fi

        if [ $? -ne 0 ]; then
            showdiag
            ret=1
        fi

        cd ..
    fi

    if [ -d zeek-testing-private ]; then
        cd zeek-testing-private
        if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
            make
        else
            make leaks
        fi

        if [ $? -ne 0 ]; then
            showdiag
            ret=1
        fi

        cd ..
    fi

    cd ../..

    echo
    echo "Result code after running tests: $ret"

    if [ $ret -ne 0 ]; then
        COREFILES=`find testing/btest/.tmp testing/external/*/.tmp -type f -name core*`

        echo
        echo "Search for core dumps ##############################################"
        echo
        echo $COREFILES

        for cf in $COREFILES; do
            echo
            echo "############# Begin stack trace for $cf ###############"
            gdb build/src/zeek -c "$cf" -ex "thread apply all bt" -ex "set pagination 0" -batch;
            echo "############# End stack trace for $cf #################"
            echo
        done
    fi

    # If we get here, then external tests were successful.
    exit $ret
}

# Show failed tests (not skipped tests) from diag.log when a test fails.
showdiag() {
    f=diag.log

    grep -qs '... failed$' $f && \
      echo && \
      echo "Output of failed external tests #####################################" && \
      echo && \
      grep -v "... not available, skipped" $f
}

# Remove the docker container.
remove_container() {
    echo "Removing the docker container..."
    docker rm -f zeektest > /dev/null
}


if [ ! -f ci/travis-job ]; then
    echo "Error: must change directory to root of zeek source tree before running this script."
    exit 1
fi

set -e

if [ "${TRAVIS_EVENT_TYPE}" = "cron" ] || [ "$distro" = "coverity" ]; then
    # Check if the project token is available (this is a secret value and
    # should not be hard-coded in this script).  This value can be found by
    # logging into the coverity scan web site and looking in the project
    # settings.
    if [ -z "${COV_TOKEN}" ]; then
        echo "Error: COV_TOKEN is not defined (should be defined in environment variables section of Travis settings for this repo)"
        exit 1
    fi
fi

if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
    # This is a Travis CI cron job which only runs the Coverity build inside
    # a single container.

    if [ "$distro" != "${COVERITY_DISTRO}" ]; then
        echo "Coverity scan is performed on ${COVERITY_DISTRO}"
        exit 0
    fi
fi

if [ "$distro" = "coverity" ]; then
    # The "build" and "run" steps are split up into separate steps because the
    # build outputs thousands of lines (which are conveniently collapsed into
    # a single line when viewing the "Job log" on the Travis CI web site).
    if [ "$step" = "install" ]; then
        echo "No 'install' step needed for Coverity build"
    elif [ "$step" = "build" ]; then
        install_coverity
        build_coverity
    elif [ "$step" = "run" ]; then
        run_coverity
    elif [ "$step" = "all" ]; then
        install_coverity
        build_coverity
        run_coverity
    fi
elif [ "$distro" = "travis" ]; then
    # Build Zeek and run tests.

    # The "build" and "run" steps are split up into separate steps because the
    # build outputs thousands of lines (which are conveniently collapsed into
    # a single line when viewing the "Job log" on the Travis CI web site).
    if [ "$step" = "build" ]; then
        build
    elif [ "$step" = "run" ]; then
        run
    elif [ "$step" = "all" ]; then
        build
        run
    fi
else
    # Build Zeek and run tests in a docker container.

    if [ "$step" = "install" ]; then
        install_in_docker
    elif [ "$step" = "build" ]; then
        build_in_docker
    elif [ "$step" = "run" ]; then
        run_in_docker
    elif [ "$step" = "all" ]; then
        install_in_docker
        build_in_docker
        run_in_docker
        # If all tests pass, then remove the docker container.
        remove_container
    fi
fi
